{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "78c4e3a4",
   "metadata": {},
   "source": [
    "## DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ea551d48",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T02:04:56.287186Z",
     "start_time": "2024-05-11T02:04:51.246562Z"
    }
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "from IPython import display\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3408c33e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T02:04:58.165620Z",
     "start_time": "2024-05-11T02:04:58.157099Z"
    }
   },
   "outputs": [],
   "source": [
    "class GymHelper:\n",
    "    def __init__(self,env,figsize=(3,3)):\n",
    "        self.env=env\n",
    "        self.figsize=figsize\n",
    "        plt.figure(figsize=figsize)\n",
    "        self.img=plt.imshow(env.render())\n",
    "    def render(self,title=None):\n",
    "        img_data=self.env.render()\n",
    "        self.img.set_data(img_data)\n",
    "        display.display(plt.gcf())\n",
    "        display.clear_output(wait=True)\n",
    "        if title:\n",
    "            plt.title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dd08a077",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T02:05:02.879560Z",
     "start_time": "2024-05-11T02:05:00.054555Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADbCAYAAADNoUzuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAZx0lEQVR4nO3dfVBU590+8GsXdpe33Q0vcY8rYNBArEFtxWhlrGBEjAmxTmeqralJZpyMNkLkp05Gm7ZiWoGaRtOOVaet1c70scSM0tjWUklFGodkohgi6lPTF+RNNojC7oLLLuzevz/yeJoVUBdhb5DrM3P+OPe59+z3HNhrz97nnF2NEEKAiEgCrewCiGjsYgARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAkZ+3334bjz/+OMLDw6HRaLBs2TJoNJpBrevUqVPQaDQ4depUQI975JFHkJOTM6jnDMSVK1eg0Whw8ODBYX8u6l+o7AJo5Lh27RpWrVqFp556Cnv27IHBYIDVasXmzZsHtb6ZM2figw8+wNSpU4e4UnpQMIBI9emnn6Knpwff+c53kJGRobYnJiYOan0mkwlf/epXh6o8egDxIxgBAF588UXMmzcPALBixQpoNBpkZmaioKCgz0ewWx+RysrKMHPmTISHh2PKlCn4zW9+49evv49g//nPf/Ctb30LVqsVBoMBFosFCxcuRE1NTZ+a7rZ+ALDZbFizZg3i4+Oh1+uRlJSEbdu2obe316/f1atXsXz5chiNRpjNZqxYsQI2m22Qe4uGCo+ACADwgx/8ALNnz8a6detQWFiIBQsWwGQy4fDhw/32/+STT7Bx40Zs3rwZFosFv/71r7F69Wo8+uijmD9//oDP8/TTT8Pr9WLHjh1ITExEW1sbqqqq0NHREfD6bTYbZs+eDa1Wix/+8IeYPHkyPvjgA/z4xz/GlStXcODAAQCAy+VCVlYWrl69iqKiIqSkpODPf/4zVqxYMTQ7jwZPEP2fiooKAUC88847atvWrVvF7f8mEydOFGFhYaK+vl5tc7lcIiYmRqxZs6bP+ioqKoQQQrS1tQkA4q233rpjHfe6/jVr1oioqCi/fkII8dOf/lQAEBcvXhRCCLF3714BQLz77rt+/V566SUBQBw4cOCO9dDw4UcwGpQvf/nLfmNDYWFhSElJQX19/YCPiYmJweTJk/HGG29g586d+Pjjj+Hz+Qa9/j/96U9YsGABrFYrent71WnJkiUAgMrKSgBARUUFjEYjli5d6vccK1euDHzDaUgxgGhQYmNj+7QZDAa4XK4BH6PRaPC3v/0Nixcvxo4dOzBz5kw8/PDDeOWVV+B0OgNe/2effYY//vGP0Ol0ftPjjz8OAGhrawMAXL9+HRaLpc/6FEW5t42lYcMxIAqqiRMnYv/+/QA+P+t2+PBhFBQUwOPxYN++fQGtKy4uDtOnT8f27dv7XW61WgF8HmYfffRRn+UchJaPAUTSpKSk4Pvf/z6OHDmCc+fOBfz4nJwcHD9+HJMnT0Z0dPSA/RYsWIDDhw/j2LFjfh/DDh06NKi6aegwgChozp8/j9zcXHzzm99EcnIy9Ho9Tp48ifPnzw/qYsfXX38d5eXlSE9PxyuvvILHHnsM3d3duHLlCo4fP459+/YhPj4ezz//PHbt2oXnn38e27dvR3JyMo4fP46//vWvw7CVFAgGEAWNoiiYPHky9uzZg8bGRmg0GkyaNAlvvvkm8vLyAl7f+PHjcfbsWfzoRz/CG2+8gaamJhiNRiQlJeGpp55Sj4oiIiJw8uRJrF+/Hps3b4ZGo0F2djZKSkqQnp4+1JtJAdAIwV/FICI5eBaMiKRhABGRNAwgIpJGagDt2bMHSUlJCAsLQ1paGt5//32Z5RBRkEkLoLfffhv5+fl47bXX8PHHH+NrX/salixZgoaGBlklEVGQSTsLNmfOHMycORN79+5V2770pS9h2bJlKCoqklESEQWZlOuAPB4Pqqur+1x8lp2djaqqqj793W433G63Ou/z+XDjxg3ExsYO+utCiWj4CCHgdDphtVqh1Q78QUtKALW1tcHr9fa5QdBisfR7f05RURG2bdsWrPKIaIg0NjYiPj5+wOVSr4S+/ehFCNHvEc2WLVuwYcMGdd5utyMxMRGNjY0wmUzDXicRBcbhcCAhIQFGo/GO/aQEUFxcHEJCQvoc7bS2tvb7tQkGgwEGg6FPu8lkYgARjWB3GyKRchZMr9cjLS0N5eXlfu23biwkorFB2kewDRs2YNWqVZg1axbmzp2LX/7yl2hoaMDatWtllUREQSYtgFasWIHr16/j9ddfR0tLC1JTU3H8+HFMnDhRVklEFGSj8m54h8MBs9kMu93OMSCiEeheX6O8F4yIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSBBxAf//73/Hss8/CarVCo9HgD3/4g99yIQQKCgpgtVoRHh6OzMxMXLx40a+P2+1GXl4e4uLiEBkZiaVLl6Kpqem+NoSIRp+AA6irqwszZszA7t27+12+Y8cO7Ny5E7t378aZM2egKAoWLVoEp9Op9snPz0dpaSlKSkpw+vRpdHZ2IicnB16vd/BbQkSjj7gPAERpaak67/P5hKIoori4WG3r7u4WZrNZ7Nu3TwghREdHh9DpdKKkpETt09zcLLRarSgrK7un57Xb7QKAsNvt91M+EQ2Te32NDukYUF1dHWw2G7Kzs9U2g8GAjIwMVFVVAQCqq6vR09Pj18dqtSI1NVXtczu32w2Hw+E3EdHoN6QBZLPZAAAWi8Wv3WKxqMtsNhv0ej2io6MH7HO7oqIimM1mdUpISBjKsolIkmE5C6bRaPzmhRB92m53pz5btmyB3W5Xp8bGxiGrlYjkGdIAUhQFAPocybS2tqpHRYqiwOPxoL29fcA+tzMYDDCZTH4TEY1+QxpASUlJUBQF5eXlapvH40FlZSXS09MBAGlpadDpdH59WlpacOHCBbUPEY0NoYE+oLOzE//617/U+bq6OtTU1CAmJgaJiYnIz89HYWEhkpOTkZycjMLCQkRERGDlypUAALPZjNWrV2Pjxo2IjY1FTEwMNm3ahGnTpiErK2votoyIRr5AT69VVFQIAH2mF154QQjx+an4rVu3CkVRhMFgEPPnzxe1tbV+63C5XCI3N1fExMSI8PBwkZOTIxoaGu65Bp6GJxrZ7vU1qhFCCIn5NygOhwNmsxl2u53jQUQj0L2+RnkvGBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0Af8uGNFw8Xq60XrpFAzGOIQ9pEAf+RBCDJF3/VlvGr0YQDRi3LzeiKvVf4KvtwfaUD10kWY88rXvwDRhiuzSaJgwgGhEEELA1X4Vvl4PAMDX60avy4nQsEjJldFw4hgQjRj2hgt+86HhRhiMD0uqhoKBAUQjgtd9E92OVr824/gUaEP1kiqiYGAA0YjQc9MOt6PtCy0aRMQmQKPlv+iDjH9dGhG62uohfF51XqPVIsoySWJFFAwBBVBRURGeeOIJGI1GjBs3DsuWLcPly5f9+gghUFBQAKvVivDwcGRmZuLixYt+fdxuN/Ly8hAXF4fIyEgsXboUTU1N9781NCoJIXCzrQEQPrVNF/EQdJEPySuKgiKgAKqsrMS6devw4Ycfory8HL29vcjOzkZXV5faZ8eOHdi5cyd2796NM2fOQFEULFq0CE6nU+2Tn5+P0tJSlJSU4PTp0+js7EROTg68Xm9/T0sPOOHtgaPZ/43MYH4YunCjpIooaMR9aG1tFQBEZWWlEEIIn88nFEURxcXFap/u7m5hNpvFvn37hBBCdHR0CJ1OJ0pKStQ+zc3NQqvVirKysnt6XrvdLgAIu91+P+XTCOHq+EycO/j/xEf7XlKnq+eOC5/PJ7s0GqR7fY3e1xiQ3W4HAMTExAAA6urqYLPZkJ2drfYxGAzIyMhAVVUVAKC6uho9PT1+faxWK1JTU9U+t3O73XA4HH4TPTi6O1rQ292pzmu0oQiPjecV0GPAoANICIENGzZg3rx5SE1NBQDYbDYAgMVi8etrsVjUZTabDXq9HtHR0QP2uV1RURHMZrM6JSQkDLZsGoG6rtX7zYfowxARGy+pGgqmQQdQbm4uzp8/j9///vd9lt3+ziWEuOu72Z36bNmyBXa7XZ0aGxsHWzaNMMLn7RNABtPDCNFHSKqIgmlQAZSXl4djx46hoqIC8fH/fadSFAUA+hzJtLa2qkdFiqLA4/Ggvb19wD63MxgMMJlMfhM9GHpczj4BFBGXyAsQx4iAAkgIgdzcXBw9ehQnT55EUlKS3/KkpCQoioLy8nK1zePxoLKyEunp6QCAtLQ06HQ6vz4tLS24cOGC2ofGjp6bdvh6ur/QooFpwhSO/4wRAd2Mum7dOhw6dAjvvvsujEajeqRjNpsRHh4OjUaD/Px8FBYWIjk5GcnJySgsLERERARWrlyp9l29ejU2btyI2NhYxMTEYNOmTZg2bRqysrKGfgtpxBJCoNP2L/UGVADQaENgMMZKrIqCKaAA2rt3LwAgMzPTr/3AgQN48cUXAQCvvvoqXC4XXn75ZbS3t2POnDk4ceIEjMb/XtOxa9cuhIaGYvny5XC5XFi4cCEOHjyIkJCQ+9saGl2EQNe1K35NBvM4hD00Xk49FHQaIYSQXUSgHA4HzGYz7HY7x4NGMW+PG5dKC9Hd3qK2RU9Kw+SFL/EesFHuXl+j/CuTNN0dtttuQAWMyqMAx3/GDAYQSePpvAHh7VHntaF6RFomcwB6DGEAkRRCCDia/9evTRuqh543oI4pDCCSwtfrwc0bzX5tkeOSEMobUMcUBhBJ4XXfRPeNq35tETEToNHwX3Is4V+bpOhs/Q96PS6/tqjxKRz/GWMYQBR0QojPz375fQGZGeHRvP5nrGEAkQR9B6BDDZH8ArIxiAFEQdfb3QW345pfm2nCl6AJ4c/UjTUMIAo6t/M63M7r/23QaBAeOwEAx3/GGgYQBZ2z+X/9xn+0ITpE8QLEMYkBREElhA/dt338CjNbeAHiGMUAoqDy9XjQafu3X5s+KhpaXZikikgmBhAFlafrBnpcdr82U/zjkqoh2RhAFDRCCNy83gSv+6bapgkJ5fU/YxgDiILK0XTb9T9hRv4EzxjGAKKgEcLX5/qfyLhEhHD8Z8xiAFHQ9HR1wNXufwNq2EMWaLT8Kt6xigFEQdNtb4X3izegarQwWvkLGGMZA4iC4vNfwPgnhM+rtoXowxFmHiexKpKNAUTBIXxw3fb9P2Hmh6GPipFUEI0EDCAKCm+PGzfbGvzawswKx3/GOAYQBYWr/So8Nzv82ozjU+QUQyMGA4iCoru9BcLbq86H6MMROe4RDkCPcfwCFrpvvb296OzsvGOf61dq/eZFaBhcvVp4Ojr69NXr9YiIiBjKEmmEYgDRfTt//jyWLVsGn8/X7/IIgw6vP/9VTLJ8/o2HGg3wXtUn2L1lNvr7Wd7nnnsOP/nJT4axYhopGEB03zweD5qbmwcMIGucCd6wx/GRYxqMITcwOeIc/tnQiqbm5n77d/RzVEQPpoDGgPbu3Yvp06fDZDLBZDJh7ty5+Mtf/qIuF0KgoKAAVqsV4eHhyMzMxMWLF/3W4Xa7kZeXh7i4OERGRmLp0qVoamoamq2hEUmJfwL/9jyJGz0TUN+diovOdJz91Ca7LBoBAgqg+Ph4FBcX4+zZszh79iyefPJJfP3rX1dDZseOHdi5cyd2796NM2fOQFEULFq0CE6nU11Hfn4+SktLUVJSgtOnT6OzsxM5OTnwer0DPS2NcuOViYBW/39zGvz7mg4Nnzmk1kQjQ0AB9Oyzz+Lpp59GSkoKUlJSsH37dkRFReHDDz+EEAJvvfUWXnvtNXzjG99Aamoqfvvb3+LmzZs4dOgQAMBut2P//v148803kZWVha985Sv43e9+h9raWrz33nvDsoEkV4hWgyeneGHADQACWvRA03kO9i7XXR9LD75BjwF5vV6888476Orqwty5c1FXVwebzYbs7Gy1j8FgQEZGBqqqqrBmzRpUV1ejp6fHr4/VakVqaiqqqqqwePHigGr4xz/+gaioqMFuAg2Rurq6AZd5fQK7/qcUCRM+genhGZiRGIJ/fvrhgONFANDe3o5Lly4NR6kUJHc7K3pLwAFUW1uLuXPnoru7G1FRUSgtLcXUqVNRVVUFALBYLH79LRYL6uvrAQA2mw16vR7R0dF9+thsA48JuN1uuN1udd7h+Pzw3W63o7e3d6CHUZDc7Z+t6ZoDTddqANTgPa0Gor9TX1/g8Xg4ED3KdXV13VO/gAPoscceQ01NDTo6OnDkyBG88MILqKysVJfffmGZEOKuF5vdrU9RURG2bdvWp33OnDkwmUwBbgENNa323j/Je313SR98/oaUnp5+PyWRZLcOEu4m4Cuh9Xo9Hn30UcyaNQtFRUWYMWMGfvazn0FRFADocyTT2tqqHhUpigKPx4P29vYB+/Rny5YtsNvt6tTY2Bho2UQ0At33rRhCCLjdbiQlJUFRFJSXl6vLPB4PKisr1XeztLQ06HQ6vz4tLS24cOHCHd/xDAaDeur/1kREo19AH8G+973vYcmSJUhISIDT6URJSQlOnTqFsrIyaDQa5Ofno7CwEMnJyUhOTkZhYSEiIiKwcuVKAIDZbMbq1auxceNGxMbGIiYmBps2bcK0adOQlZU1LBtIRCNXQAH02WefYdWqVWhpaYHZbMb06dNRVlaGRYsWAQBeffVVuFwuvPzyy2hvb8ecOXNw4sQJGI1GdR27du1CaGgoli9fDpfLhYULF+LgwYMICeHXMoxWISEhMJlMdzyzFYiwMH5H9FihEeJu5yRGHofDAbPZDLvdzo9jI4Db7UZra+uQrS8yMhIxMfyistHsXl+jvBeM7pvBYEBCQoLsMmgU4vcBEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImlCZRcwGEIIAIDD4ZBcCRH159Zr89ZrdSCjMoCcTicAICEhQXIlRHQnTqcTZrN5wOUacbeIGoF8Ph8uX76MqVOnorGxESaTSXZJo4LD4UBCQgL3WYC43wInhIDT6YTVaoVWO/BIz6g8AtJqtZgwYQIAwGQy8Z8iQNxng8P9Fpg7HfncwkFoIpKGAURE0ozaADIYDNi6dSsMBoPsUkYN7rPB4X4bPqNyEJqIHgyj9giIiEY/BhARScMAIiJpGEBEJM2oDKA9e/YgKSkJYWFhSEtLw/vvvy+7JGmKiorwxBNPwGg0Yty4cVi2bBkuX77s10cIgYKCAlitVoSHhyMzMxMXL1706+N2u5GXl4e4uDhERkZi6dKlaGpqCuamSFNUVASNRoP8/Hy1jfssSMQoU1JSInQ6nfjVr34lLl26JNavXy8iIyNFfX297NKkWLx4sThw4IC4cOGCqKmpEc8884xITEwUnZ2dap/i4mJhNBrFkSNHRG1trVixYoUYP368cDgcap+1a9eKCRMmiPLycnHu3DmxYMECMWPGDNHb2ytjs4Lmo48+Eo888oiYPn26WL9+vdrOfRYcoy6AZs+eLdauXevXNmXKFLF582ZJFY0sra2tAoCorKwUQgjh8/mEoiiiuLhY7dPd3S3MZrPYt2+fEEKIjo4OodPpRElJidqnublZaLVaUVZWFtwNCCKn0ymSk5NFeXm5yMjIUAOI+yx4RtVHMI/Hg+rqamRnZ/u1Z2dno6qqSlJVI4vdbgcAxMTEAADq6upgs9n89pnBYEBGRoa6z6qrq9HT0+PXx2q1IjU19YHer+vWrcMzzzyDrKwsv3bus+AZVTejtrW1wev1wmKx+LVbLBbYbDZJVY0cQghs2LAB8+bNQ2pqKgCo+6W/fVZfX6/20ev1iI6O7tPnQd2vJSUlOHfuHM6cOdNnGfdZ8IyqALpFo9H4zQsh+rSNRbm5uTh//jxOnz7dZ9lg9tmDul8bGxuxfv16nDhxAmFhYQP24z4bfqPqI1hcXBxCQkL6vMO0trb2ebcaa/Ly8nDs2DFUVFQgPj5ebVcUBQDuuM8URYHH40F7e/uAfR4k1dXVaG1tRVpaGkJDQxEaGorKykr8/Oc/R2hoqLrN3GfDb1QFkF6vR1paGsrLy/3ay8vLkZ6eLqkquYQQyM3NxdGjR3Hy5EkkJSX5LU9KSoKiKH77zOPxoLKyUt1naWlp0Ol0fn1aWlpw4cKFB3K/Lly4ELW1taipqVGnWbNm4bnnnkNNTQ0mTZrEfRYsEgfAB+XWafj9+/eLS5cuifz8fBEZGSmuXLkiuzQpvvvd7wqz2SxOnTolWlpa1OnmzZtqn+LiYmE2m8XRo0dFbW2t+Pa3v93vKeX4+Hjx3nvviXPnzoknn3xyTJ1S/uJZMCG4z4Jl1AWQEEL84he/EBMnThR6vV7MnDlTPeU8FgHodzpw4IDax+fzia1btwpFUYTBYBDz588XtbW1futxuVwiNzdXxMTEiPDwcJGTkyMaGhqCvDXy3B5A3GfBwa/jICJpRtUYEBE9WBhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNP8fvHjc6aTtvSYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "env=gym.make(\"CartPole-v1\",render_mode=\"rgb_array\")\n",
    "env.reset()\n",
    "gym_helper=GymHelper(env)\n",
    "i=0\n",
    "while True:\n",
    "    gym_helper.render(title=str(i))\n",
    "    action=env.action_space.sample()\n",
    "    observation,reward,terminated,truncated,info=env.step(action)\n",
    "    done=terminated or truncated\n",
    "    i+=1\n",
    "    if done:\n",
    "        break\n",
    "gym_helper.render(\"finished\")\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ea6bec31",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T02:05:09.397313Z",
     "start_time": "2024-05-11T02:05:04.890313Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from tqdm import *\n",
    "import collections\n",
    "import time\n",
    "import random\n",
    "import sys\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "71ff1b51",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T03:30:41.936830Z",
     "start_time": "2024-05-11T03:30:41.928790Z"
    }
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self,input_dim,output_dim):\n",
    "        super(Net,self).__init__()\n",
    "        self.input_dim=input_dim\n",
    "        self.output_dim=output_dim\n",
    "        self.fc=nn.Sequential(\n",
    "            nn.Linear(self.input_dim,64),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(64,128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128,self.output_dim)\n",
    "        )\n",
    "        #dueling networks\n",
    "    def forward(self,state):\n",
    "        action=self.fc(state)\n",
    "        return action\n",
    "#dueling DQN\n",
    "class Net1(nn.Module):\n",
    "    def __init__(self,input_dim,output_dim):\n",
    "        super(Net,self).__init__()\n",
    "        self.input_dim=input_dim\n",
    "        self.output_dim=output_dim\n",
    "        self.advantage=nn.Sequential(\n",
    "            nn.Linear(64,128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128,self.output_dim)\n",
    "        )\n",
    "        self.value=nn.Sequential([\n",
    "            nn.Linear(64,128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128,1)\n",
    "        ])\n",
    "        #dueling networks\n",
    "    def forward(self,x):\n",
    "        x=self.fc(x)\n",
    "        x=x.view(x.size(0),-1)\n",
    "        advantage=self.advantage(x)\n",
    "        value=self.value(x)\n",
    "        action=value+advantage-advantage.mean()\n",
    "        return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3df5e90e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T03:30:42.550520Z",
     "start_time": "2024-05-11T03:30:42.545385Z"
    }
   },
   "outputs": [],
   "source": [
    "class Replay_buffer:\n",
    "    def __init__(self,max_size):\n",
    "        self.max_size=max_size\n",
    "        self.buffer=collections.deque(maxlen=self.max_size)\n",
    "    def add(self,state,action,reward,next_state,done):\n",
    "        experience=(state,action,reward,next_state,done)\n",
    "        self.buffer.append(experience)\n",
    "    def sample(self,batch_size):\n",
    "        batch=random.sample(self.buffer,batch_size)\n",
    "        state,action,reward,next_state,done=zip(*batch)\n",
    "        return state,action,reward,next_state,done\n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7aaaa35d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T03:34:04.934141Z",
     "start_time": "2024-05-11T03:34:04.920760Z"
    }
   },
   "outputs": [],
   "source": [
    "class DQN:\n",
    "    def __init__(self,env,lr=0.001,gamma=0.99,buffer_size=10000,T=10):\n",
    "        self.env=env\n",
    "        self.lr=lr\n",
    "        self.replay_buffer=Replay_buffer(max_size=buffer_size)\n",
    "        self.T=T\n",
    "        #判断可用设备是CPU与GPU\n",
    "        self.device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "        #定义Q网络与目标网络,模型结构是一样的\n",
    "        self.model=Net(env.observation_space.shape[0],env.action_space.n).to(self.device)\n",
    "        self.target_model=Net(env.observation_space.shape[0],env.action_space.n).to(self.device)\n",
    "        #初始化令目标网络dde的参数等于Q网络的参数\n",
    "        for param,target_param in zip(self.model.parameters(),self.target_model.parameters()):\n",
    "            target_param.data.copy_(param)\n",
    "        self.optimizer=torch.optim.Adam(self.model.parameters())\n",
    "        #记录更新次数,用于决定何时更新目标参数\n",
    "        self.update_count=0\n",
    "    def choose_action(self,state,eps=0.1):\n",
    "        if np.random.randn()<eps:\n",
    "            return np.random.randint(self.env.action_space.n)\n",
    "        else:\n",
    "            state=torch.FloatTensor(np.array([state])).to(self.device)\n",
    "            action=self.model(state).argmax().item()\n",
    "            return action\n",
    "    #计算损失函数\n",
    "    def compute_loss(self,batch):\n",
    "        #取出数据,将其转化为numpy数组\n",
    "        #然后进一步转化为tensor,并将数据转移到指定计算自愿上\n",
    "        states,actions,rewards,next_states,dones=batch\n",
    "        states=torch.FloatTensor(np.array(states)).to(self.device)\n",
    "        actions=torch.IntTensor(np.array(actions)).to(self.device)\n",
    "        rewards=torch.FloatTensor(np.array(rewards)).to(self.device)\n",
    "        next_states=torch.FloatTensor(np.array(next_states)).to(self.device)\n",
    "        dones=torch.FloatTensor(np.array(dones)).to(self.device)\n",
    "        \n",
    "        #计算当前Q值,Q网络对ddadandanq当前状态动作样本对的Q值估计\n",
    "        current_Q=self.model(states).gather(1,actions)\n",
    "        #计算目标网络对下一状态的Q值估计\n",
    "        #一起更新\n",
    "        next_model_q=self.model(next_states)\n",
    "        next_Q=self.target_model(next_states)\n",
    "        #选择下一状态最大Q值\n",
    "        max_next_Q=torch.max(next_Q,1)[0].view(-1,1)\n",
    "        #DDQN\n",
    "        max_next_Q=next_Q.gather(1,torch.max(next_model_q,1)[1],upsqueeze(1))\n",
    "        #计算期望的Q值，若达到终止状态则直接是reward\n",
    "        expected_Q=rewards+(1-done)*self.gamma*max_next_Q\n",
    "        #计算当前Q值和q期望Q值的均方误差,返回结果\n",
    "        loss=torch.mean(F.mse_loss(current_Q,expected_Q))\n",
    "        return loss\n",
    "    #模型更新\n",
    "    def update(self,batch_size):\n",
    "        #从经验回放缓冲区中随机采样\n",
    "        batch=self.replay_buffer.sample(batch_size)\n",
    "        loss=self.compute_loss(batch)\n",
    "        #梯度清零,反向传播,更新参数\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optimizer.step()\n",
    "        \n",
    "        if self.update_count%self.T==0:\n",
    "            for param,target_param in zip(self.model.parameters(),self.target_model.parameters()):\n",
    "                target_param.data.copy_(param)\n",
    "        self.update_count+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4e65d2ec",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T03:34:05.791712Z",
     "start_time": "2024-05-11T03:34:05.731024Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▊                                                                                 | 2/200 [00:00<00:03, 65.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode0:11.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "gather(): Expected dtype int64 for index",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_13900\\4147774729.py\u001b[0m in \u001b[0;36m<cell line: 8>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     16\u001b[0m         \u001b[0meps_reward\u001b[0m\u001b[1;33m+=\u001b[0m\u001b[0mreward\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0magent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreplay_buffer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m>\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 18\u001b[1;33m             \u001b[0magent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     19\u001b[0m         \u001b[0mstate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnext_state\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_13900\\2511862716.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, batch_size)\u001b[0m\n\u001b[0;32m     49\u001b[0m         \u001b[1;31m#从经验回放缓冲区中随机采样\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     50\u001b[0m         \u001b[0mbatch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreplay_buffer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msample\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m         \u001b[0mloss\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompute_loss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     52\u001b[0m         \u001b[1;31m#梯度清零,反向传播,更新参数\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_13900\\2511862716.py\u001b[0m in \u001b[0;36mcompute_loss\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m     35\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m         \u001b[1;31m#计算当前Q值,Q网络对ddadandanq当前状态动作样本对的Q值估计\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 37\u001b[1;33m         \u001b[0mcurrent_Q\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mactions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     38\u001b[0m         \u001b[1;31m#计算目标网络对下一状态的Q值估计\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m         \u001b[0mnext_Q\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtarget_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnext_states\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: gather(): Expected dtype int64 for index"
     ]
    }
   ],
   "source": [
    "max_eps=200 #每次训练的episode\n",
    "max_steps=500 #每个回合的最大步数\n",
    "batch_size=32\n",
    "\n",
    "agent=DQN(env)\n",
    "eps_rewards=[]\n",
    "\n",
    "for episode in tqdm(range(max_eps)):\n",
    "    state,_=env.reset()\n",
    "    eps_reward=0\n",
    "    for step in range(max_steps):\n",
    "        action=agent.choose_action(state)\n",
    "        next_state,reward,terminated,truncated,info=env.step(action)\n",
    "        done=terminated or truncated\n",
    "        agent.replay_buffer.add(state,action,reward,next_state,done)\n",
    "        eps_reward+=reward\n",
    "        if len(agent.replay_buffer)>batch_size:\n",
    "            agent.update(batch_size)\n",
    "        state=next_state\n",
    "        if done:\n",
    "            break\n",
    "    eps_rewards.append(eps_reward)\n",
    "    if episode % 40==0:\n",
    "        tqdm.write(\"Episode\"+str(episode)+\":\"+str(eps_reward))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "598b2767",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-11T03:34:26.821620Z",
     "start_time": "2024-05-11T03:34:26.676578Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1579e416470>]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFSklEQVR4nO3dd1xT9/4/8NdJAmEIEUTDdONW3CijautoUdS21tU6a92Ctb2OW1vtbatX23or4qqzw1UHuGirHWpYKirWvUDBgQgIYYaR8/ujv/ItdQECJ+P1fDzOHznJOb7yeVjz6nlnCKIoiiAiIiIyYDKpAxARERE9CwsLERERGTwWFiIiIjJ4LCxERERk8FhYiIiIyOCxsBAREZHBY2EhIiIig8fCQkRERAZPIXWAqqLX63H37l3Y2dlBEASp4xAREVE5iKKI7OxsuLq6QiZ78nUUkyksd+/ehYeHh9QxiIiIqBKSk5Ph7u7+xPtNprDY2dkB+PMJ29vbS5yGiIiIykOr1cLDw6P0dfxJTKaw/DUGsre3Z2EhIiIyMs96OwffdEtEREQGj4WFiIiIDB4LCxERERk8FhYiIiIyeCwsREREZPBYWIiIiMjgsbAQERGRwWNhISIiIoPHwkJEREQGr8KF5dixYwgMDISrqysEQUB4eHiZ+/fs2YN+/frByckJgiAgPj6+XOfdvXs3WrVqBaVSiVatWiEsLKyi0YiIiMhEVbiw5ObmwsvLC6GhoU+839fXF//973/Lfc6YmBgMGzYMo0aNwtmzZzFq1CgMHToUx48fr2g8IiIiMkGCKIpipQ8WBISFhWHw4MGP3Hfz5k00atQIZ86cQfv27Z96nmHDhkGr1eLHH38s3ffyyy/DwcEB27ZtK1cWrVYLlUqFrKws/pYQERGRkSjv67dBvIclJiYGffv2LbOvX79+iI6OfuIxOp0OWq22zFYdfr6QgqBtZ5BdUFQt5yciIqJnM4jCkpKSArVaXWafWq1GSkrKE49ZvHgxVCpV6ebh4VHlufILS/DvPeew7+xdBK6IxPk7WVX+ZxAREdGzGURhAR79WWlRFJ/6U9Pz5s1DVlZW6ZacnFzlmawt5fh6dCe4qqxwMz0Pr62KxncxN/EcUzQiIiKqBIMoLM7Ozo9cTUlNTX3kqsvfKZVK2Nvbl9mqQ6cGjjgY5I/eLeuhsESPD/dewLStp6HliIiIiKjGGERh6d69Ow4fPlxm36FDh+Dj4yNRorIcbC2xbnRnzO/fEgqZgIhzKegfosHZ5EypoxEREZkFRUUPyMnJwfXr10tvJyYmIj4+Ho6Ojqhfvz4yMjKQlJSEu3fvAgCuXLkC4M+rKM7OzgCA0aNHw83NDYsXLwYABAcH44UXXsCSJUswaNAg7N27F7/88gsiIyOf+wlWFUEQMMG/MTo3dMT0raeRnJGPIWuiMe+Vlhjn2/Cp4ysiIiJ6PhW+whIXF4cOHTqgQ4cOAIBZs2ahQ4cO+OijjwAA+/btQ4cOHdC/f38AwPDhw9GhQwesWbOm9BxJSUm4d+9e6W0fHx9s374dmzZtQrt27bB582bs2LED3t7ez/XkqkN7j9o4GOSPfq3VKCoR8Z8DFzHxu1PIzCuUOhoREZHJeq7vYTEkNf09LKIo4tuYW/js4CUUlujhVtsaK0Z2QMf6DtX+ZxMREZkKo/oeFmMkCALG+DTE7ik+aFDHBncy8zF0TQy+PnYDer1JdEAiIiKDwcLynNq6q7B/hh/6t3NBsV7EoojLmPBtHDJyOSIiIiKqKiwsVcDeygKhIzrgs1fbwFIhw2+XU9E/RIOTNzOkjkZERGQSWFiqiCAIeNO7AcKn+qKxky3uZRVg+NexWPn7dY6IiIiInhMLSxVr5WqPfTP8MLi9K0r0Ij7/+QrGbDqBtByd1NGIiIiMFgtLNailVOB/w9pj6evtYGUhg+ZaGgKWaxBzI13qaEREREaJhaWaCIKAoV08sHeaH5rWq4XUbB3eXB+L5b9cQwlHRERERBXCwlLNmjvbYd90Xwzp5A69CPzvl6sYteE4UrMLpI5GRERkNFhYaoCNpQJfvOGFL9/wgrWFHNE30hGwXIPIa2lSRyMiIjIKLCw16PVO7tg/ww/N1XZIyynEqI3H8eWhKygu0UsdjYiIyKCxsNSwpvVqYe90X4zo6gFRBFb8dh0j1x9HShZHRERERE/CwiIBKws5Fr/WDsuHt4etpRwnEjMQEKLBkSupUkcjIiIySCwsEhrU3g37Z/ihlYs9MnILMXbTSfz3x8so4oiIiIioDBYWiTWuWwt7pvpgVLcGAIA1R29g+NexuJuZL3EyIiIiw8HCYgCsLOT4ZHAbrHqzI+yUCpy69RABIRr8cvG+1NGIiIgMAguLAQlo64KDQf5o565CZl4RJnwbh08PXERhMUdERERk3lhYDEz9OjbYObk7xvk2BACsj0zEG2tjkJyRJ20wIiIiCbGwGCClQo4Fga2xdlQn2FspcDY5E/1DNPjpfIrU0YiIiCTBwmLA+rV2RkSwP9p71Ia2oBiTvz+FhfsuQFdcInU0IiKiGsXCYuDcHf4cEU18oTEAYHP0TQxZHYNb6bkSJyMiIqo5LCxGwEIuw78DWmLj2M6obWOBc3ey0D8kEgf+uCt1NCIiohrBwmJEXmyhRkSQPzo3cECOrhjTt57BB2HnUFDEEREREZk2FhYj41rbGtsndsPUnk0AAFuOJ+HVVdFIeJAjcTIiIqLqw8JihBRyGWa/3ALfjO+KOraWuHRPiwErIhF+5o7U0YiIiKoFC4sR69GsLiKC/dGtsSPyCkswc0c85uz6A/mFHBEREZFpYWExcmp7K2yZ0A1BL3lCEIAdcckYvDIK11OzpY5GRERUZVhYTIBcJmBWn2b4/m1vONVS4sr9bASuiMKuU7eljkZERFQlWFhMiG9TJ0QE+8G3aR3kF5Xg/Z1nMeuHeOQVFksdjYiI6LmwsJiYenZW+Ha8N97r0wwyAdhz+g4CV0TicopW6mhERESVxsJiguQyATNe8sTWd7pBba/EjQe5GBQahe0nkiCKotTxiIiIKoyFxYR1a1wHEUH+6NGsLnTFeszdcw7B2+ORo+OIiIiIjAsLi4mrU0uJTWO7YM7LLSCXCdh39i4CV0Tiwt0sqaMRERGVGwuLGZDJBEzp2QQ7JnaDi8oKiWm5eHVVNL6LvcURERERGQUWFjPSuaEjIoL88VKLeigs1uPD8POYvvUMtAVFUkcjIiJ6KhYWM+Nga4n1Yzpjfv+WUMgEHDx3DwNCIvHH7UypoxERET0RC4sZEgQBE/wbY+fk7nCrbY2kjDy8vjoam6ISOSIiIiKDxMJixjrUd0BEkD/6tlKjqETEx/svYtJ3p5CVxxEREREZFhYWM6eyscDaUZ2wMLAVLOUyHLp4HwEhGpxJeih1NCIiolIsLARBEDDWtxF2T/FBfUcb3MnMxxtrYrDuWAL0eo6IiIhIeiwsVKqtuwoHgvzQv60LivUiPou4hAnfxuFhbqHU0YiIyMyxsFAZ9lYWCB3ZAZ8ObgNLhQy/XU5FQIgGcTczpI5GRERmjIWFHiEIAt7q1gBhU33QyMkW97IKMOzrWKw6cp0jIiIikgQLCz1Ra1cV9s/ww6D2rijRi1j60xWM3XwSaTk6qaMREZGZYWGhp6qlVOCrYe2x5PW2UCpkOHb1AQKWaxCbkC51NCIiMiMsLPRMgiBgWJf62DfdD03r1UJqtg4j18Ui5NdrKOGIiIiIagALC5Vbc2c77Jvui9c7ukMvAssOX8XojceRml0gdTQiIjJxLCxUITaWCnw51AtfvOEFaws5oq6nI2B5JKKup0kdjYiITBgLC1XKkE7u2D/DF83VdkjL0eGtDcex7NAVjoiIiKhasLBQpTWtZ4fwab4Y3sUDogiE/HYdI9fF4r6WIyIiIqpaLCz0XKwt5fjv6+2wfHh72FrKcTwxA68s1+DIlVSpoxERkQlhYaEqMai9G/bP8ENLF3tk5BZi7KaTWPLTZRSX6KWORkREJoCFhapM47q1EDbVB6O6NQAArD5yA8O/jsXdzHyJkxERkbFjYaEqZWUhxyeD22DlyI6wUyoQd+shAkI0+PXSfamjERGREWNhoWrRv50LDgT5oa2bCpl5RXj7mzh8dvAiCos5IiIioopjYaFq06COLXZN6Y6xPg0BAOs0iRi6NgbJGXnSBiMiIqPDwkLVSqmQY+HA1lg7qhPsrRSIT85E/xANfr6QInU0IiIyIiwsVCP6tXbGwSB/tPeoDW1BMSZ9dwoL912ArrhE6mhERGQEKlxYjh07hsDAQLi6ukIQBISHh5e5XxRFLFy4EK6urrC2tkbPnj1x4cKFZ573q6++QvPmzWFtbQ0PDw+8++67KCjgF5CZEg9HG/wwqTve8W8EANgcfRNDVsfgVnquxMmIiMjQVbiw5ObmwsvLC6GhoY+9f+nSpVi2bBlCQ0Nx8uRJODs7o0+fPsjOzn7iObds2YK5c+diwYIFuHTpEjZs2IAdO3Zg3rx5FY1HBs5SIcMH/Vthw5jOqG1jgXN3sjAgJBIH/7gndTQiIjJggiiKlf7xF0EQEBYWhsGDBwP48+qKq6srZs6ciTlz5gAAdDod1Go1lixZgkmTJj32PNOnT8elS5fw66+/lu577733cOLECWg0mnJl0Wq1UKlUyMrKgr29fWWfEtWgu5n5CNp2BnG3HgIA3upWH/P7t4KVhVziZEREVFPK+/pdpe9hSUxMREpKCvr27Vu6T6lUokePHoiOjn7icX5+fjh16hROnDgBAEhISEBERAT69+//xGN0Oh20Wm2ZjYyLa21rbJvYDVN7NgEAfB+bhFdXRSPhQY7EyYiIyNBUaWFJSfnzkx9qtbrMfrVaXXrf4wwfPhyffPIJ/Pz8YGFhgSZNmqBXr16YO3fuE49ZvHgxVCpV6ebh4VE1T4JqlIVchtkvt8A347vC0dYSl+5pEbgiEnvj70gdjYiIDEi1fEpIEIQyt0VRfGTf3x05cgSfffYZVq1ahdOnT2PPnj04cOAAPvnkkyceM2/ePGRlZZVuycnJVZafal6PZnXxY7A/vBs5IrewBMHb4zF39x/IL+SniIiICFBU5cmcnZ0B/HmlxcXFpXR/amrqI1dd/u7DDz/EqFGjMGHCBABA27ZtkZubi4kTJ+KDDz6ATPZor1IqlVAqlVUZnySmtrfClgneCPn1Glb8fh3bTybjTFImVr7ZAU3r2Ukdj4iIJFSlV1gaNWoEZ2dnHD58uHRfYWEhjh49Ch8fnycel5eX90gpkcvlEEURz/GeYDJCCrkMs/o2x3fjveFUS4kr97MRuCIKu07dljoaERFJqMKFJScnB/Hx8YiPjwfw5xtt4+PjkZSUBEEQMHPmTCxatAhhYWE4f/48xo4dCxsbG4wcObL0HKNHjy7zkeXAwECsXr0a27dvR2JiIg4fPowPP/wQAwcOhFzOT4yYIz9PJ0QE+8G3aR3kF5Xg/Z1n8d4PZ5FXWCx1NCIikkCFR0JxcXHo1atX6e1Zs2YBAMaMGYPNmzdj9uzZyM/Px9SpU/Hw4UN4e3vj0KFDsLP7v0v6SUlJZa6ozJ8/H4IgYP78+bhz5w7q1q2LwMBAfPbZZ8/z3MjI1bOzwrfjvbHy9+v46per2H36Ns7ezsTKkR3R3JkjIiIic/Jc38NiSPg9LKYtNiEdwdvP4L5WB6VChv8Mao2hnT2e+mZuIiIyfJJ8DwtRdenWuA4igvzxQrO60BXrMWf3Oby7Ix45Oo6IiIjMAQsLGY06tZTYPLYLZr/cHHKZgPD4uxi4IhIX7/JLA4mITB0LCxkVmUzA1J5NsWNiN7iorJCQlovBq6LwfewtfqKMiMiEsbCQUerc0BERQf54qUU9FBbrMT/8PKZvOwNtQZHU0YiIqBqwsJDRcrC1xPoxnfFBQEsoZAIO/nEPA0Iice52ltTRiIioirGwkFETBAHvvNAYP0zuDrfa1kjKyMPrq6OxOSqRIyIiIhPCwkImoWN9B0QE+aNvKzUKS/RYuP8iJn9/Cll5HBEREZkCFhYyGSobC6wd1QkLAlvBQi7g5wv30X+FBmeSHkodjYiInhMLC5kUQRAwzrcRdk/xQX1HG9x+mI831sRgvSaBIyIiIiPGwkImqZ17bRwI8kP/ti4o1ov49OAlTPgmDg9zC6WORkRElcDCQibL3soCoSM74JPBbWCpkOHXy6noH6LBqVsZUkcjIqIKYmEhkyYIAkZ1a4CwqT5o5GSLu1kFGLo2FquP3IBezxEREZGxYGEhs9DaVYX9M/wwqL0rSvQilvx0GeM2n0R6jk7qaEREVA4sLGQ2aikV+GpYe/z3tbZQKmQ4evUBAkI0OJ6QLnU0IiJ6BhYWMiuCIGB41/rYO90XTera4r5WhxHrYrHi12so4YiIiMhgsbCQWWrhbI/9M/zwekd36EXgy8NXMXrjcTzI5oiIiMgQsbCQ2bKxVODLoV744g0vWFvIEXU9Ha8s1yDqeprU0YiI6B9YWMjsDenkjn3TfdFcbYe0HB3e2nAcyw5f5YiIiMiAsLAQAfBU2yF8mi+Gd/GAKAIhv17Dm+tjcV9bIHU0IiICCwtRKWtLOf77ejssH94etpZyxCZkIGC5BkevPpA6GhGR2WNhIfqHQe3dsH+GH1q62CM9txBjNp7A0p8uo7hEL3U0IiKzxcJC9BiN69ZC2FQfvNWtPgBg1ZEbGLEuFvey8iVORkRknlhYiJ7AykKOTwe3RejIDqilVODkzYcIWK7Bb5fvSx2NiMjssLAQPcOAdq44GOSHtm4qPMwrwvjNcVgUcQlFHBEREdUYFhaicmhQxxa7pnTHWJ+GAICvjyXgjTUxuP0wT9pgRERmgoWFqJyUCjkWDmyNNW91gr2VAvHJmQhYrsHPF1KkjkZEZPJYWIgq6OU2zjgY5A8vj9rQFhRj0nen8PH+Cygs5oiIiKi6sLAQVYKHow12TuqOd/wbAQA2Rd3EkDXRSErniIiIqDqwsBBVkqVChg/6t8L60Z1R28YCf9zOQv8QDSLO3ZM6GhGRyWFhIXpOvVupERHkj04NHJCtK8bULafxYfh5FBSVSB2NiMhksLAQVQHX2tbYPrEbpvRsAgD4LvYWXlsVjcS0XImTERGZBhYWoipiIZdhzsstsHlcFzjaWuLiPS0GhGiwN/6O1NGIiIweCwtRFevZvB4igvzRtZEjcgtLELw9HvP2/MERERHRc2BhIaoGziorbJ3gjRkvNoUgANtOJGPwyihcT82ROhoRkVFiYSGqJgq5DO/1bY7vxnvDqZYSl1OyEbgiErtP3ZY6GhGR0WFhIapmfp5OiAj2g0+TOsgvKsF7O8/i/Z1nkVdYLHU0IiKjwcJCVAPq2Vnhu7e98W7vZpAJwK5TtzEoNApX72dLHY2IyCiwsBDVELlMQHBvT2yZ0A317JS4lpqDgaGR2HEyCaIoSh2PiMigsbAQ1bDuTeogItgf/p5OKCjSY87uc3h3RzxydBwRERE9CQsLkQScainxzbiumP1yc8hlAsLj72LgikhcvKuVOhoRkUFiYSGSiEwmYGrPptg+sRtcVFZISMvF4FVR2HL8FkdERET/wMJCJLEuDR1xMMgfL7aoh8JiPT4IO4/p284gu6BI6mhERAaDhYXIADjaWmL96M74d0ALKGQCDv5xDwNWROL8nSypoxERGQQWFiIDIZMJmPhCE/wwuTvcalvjVnoeXlsVjW+ib3JERERmj4WFyMB0rO+AiCB/9GmlRmGJHgv2XcCU708jK58jIiIyXywsRAZIZWOBr0d1wkcDWsFCLuCnCynoH6JBfHKm1NGIiCTBwkJkoARBwHi/Rtg9xQf1HW1w+2E+hqyOxnpNAkdERGR2WFiIDFw799o4EOSHgLbOKNaL+PTgJbzzbRwy8wqljkZEVGNYWIiMgL2VBVaO7IhPBreBpUKGXy6lImC5BqduZUgdjYioRrCwEBkJQRAwqlsDhE31QSMnW9zNKsDQtbFYc/QG9HqOiIjItLGwEBmZ1q4q7J/hh4FerijRi/jvj5cx/puTSM/RSR2NiKjasLAQGaFaSgWWD2+Pxa+1hVIhw5ErDxAQosHxhHSpoxERVQsWFiIjJQgCRnStj73TfdGkri3ua3UYsS4Wob9d44iIiEwOCwuRkWvhbI990/3wWkc36EXgi0NXMWbTCTzI5oiIiEwHCwuRCbBVKrBsaHt8PqQdrC3k0FxLQ0CIBtHX06SORkRUJVhYiEzIG509sG+6L5qpa+FBtg5vbjiO/x2+ihKOiIjIyFW4sBw7dgyBgYFwdXWFIAgIDw8vc78oili4cCFcXV1hbW2Nnj174sKFC888b2ZmJqZNmwYXFxdYWVmhZcuWiIiIqGg8IrPnqbbD3ml+GNbZA6IILP/1Gt5cH4v72gKpoxERVVqFC0tubi68vLwQGhr62PuXLl2KZcuWITQ0FCdPnoSzszP69OmD7OzsJ56zsLAQffr0wc2bN7Fr1y5cuXIF69atg5ubW0XjEREAa0s5lgxph6+GtYeNpRyxCRkIWK7BsasPpI5GRFQpgvgcP0oiCALCwsIwePBgAH9eXXF1dcXMmTMxZ84cAIBOp4NarcaSJUswadKkx55nzZo1+Pzzz3H58mVYWFhUKotWq4VKpUJWVhbs7e0rdQ4iU5TwIAfTtp7BpXtaCAIwtWcTvNu7GRRyToSJSHrlff2u0n+xEhMTkZKSgr59+5buUyqV6NGjB6Kjo5943L59+9C9e3dMmzYNarUabdq0waJFi1BSUvLEY3Q6HbRabZmNiB7VuG4thE31wZve9SGKwMrfb2DEuljcy8qXOhoRUblVaWFJSUkBAKjV6jL71Wp16X2Pk5CQgF27dqGkpAQRERGYP38+vvzyS3z22WdPPGbx4sVQqVSlm4eHR9U8CSITZGUhx2evtkXoyA6opVTg5M2HCFiuwe+XU6WORkRULtVyTVgQhDK3RVF8ZN/f6fV61KtXD19//TU6deqE4cOH44MPPsDq1aufeMy8efOQlZVVuiUnJ1dZfiJTNaCdKw7M8EMbN3s8zCvCuM0nsTjiEopK9FJHIyJ6qiotLM7OzgDwyNWU1NTUR666/J2LiwuaNWsGuVxeuq9ly5ZISUlBYWHhY49RKpWwt7cvsxHRszV0ssXuKT4Y69MQALD2WAKGro3B7Yd50gYjInqKKi0sjRo1grOzMw4fPly6r7CwEEePHoWPj88Tj/P19cX169eh1//f/+VdvXoVLi4usLS0rMqIRARAqZBj4cDWWPNWR9hZKXAmKRP9QyJx6MKTR7dERFKqcGHJyclBfHw84uPjAfz5Rtv4+HgkJSVBEATMnDkTixYtQlhYGM6fP4+xY8fCxsYGI0eOLD3H6NGjMW/evNLbU6ZMQXp6OoKDg3H16lUcPHgQixYtwrRp057/GRLRE73cxgURQf7w8qiNrPwiTPzuFP6z/yIKizkiIiLDoqjoAXFxcejVq1fp7VmzZgEAxowZg82bN2P27NnIz8/H1KlT8fDhQ3h7e+PQoUOws7MrPSYpKQky2f91JQ8PDxw6dAjvvvsu2rVrBzc3NwQHB5d+NJqIqo+How12TuqOpT9dxvrIRGyMSkTcrQyEjuiI+nVspI5HRATgOb+HxZDwe1iInt8vF+/jvZ1nkZVfBDulAkuHtMMrbV2kjkVEJkyS72EhIuPWu5UaEcH+6NTAAdm6YkzZchof7T2PgqInfycSEVFNYGEhojLcaltj+8RumNyjCQDg25hbeH11NBLTciVORkTmjIWFiB5hIZdh7istsGlcFzjaWuLCXS0CV0Ri39m7UkcjIjPFwkJET9SreT1EBPmja0NH5OiKEbTtDObtOccRERHVOBYWInoqZ5UVtr7jjRkvNoUgANtOJGHwyihcT82ROhoRmREWFiJ6JoVchvf6Nse347vCqZYlLqdkY2BoJPacvi11NCIyEywsRFRu/p51ERHkD58mdZBXWIJZP5zFv3aeRV5hsdTRiMjEsbAQUYXUs7fCd297493ezSATgJ2nbmNQaBSu3s+WOhoRmTAWFiKqMLlMQHBvT2yZ0A317JS4lpqDgaGR+CEuGSbyXZREZGBYWIio0ro3qYOIYH/4ezqhoEiP2bv+wKwfziJXxxEREVUtFhYiei5OtZT4ZlxX/Ktfc8hlAsLO3EHgikhcuqeVOhoRmRAWFiJ6bjKZgGm9mmL7xG5wtrdCQlouBq2MwpbjtzgiIqIqwcJCRFWmS0NHRAT7o1fzuigs1uODsPOYse0MsguKpI5GREaOhYWIqpSjrSU2jOmCfwe0gEIm4MAf9xC4IhLn72RJHY2IjBgLCxFVOZlMwMQXmmDHpO5wq22Nm+l5eG1VNL6JvskRERFVCgsLEVWbTg0ccDDID31aqVFYoseCfRcwdctpZOVzREREFcPCQkTVqraNJb4e1QkfDWgFC7mAH8+nYMAKDc4mZ0odjYiMCAsLEVU7QRAw3q8Rdk32gYejNZIz8jFkTTQ2RCZyRERE5cLCQkQ1xsujNg7M8McrbZxRVCLikwMX8c63p5CZVyh1NCIycCwsRFSjVNYWWPVmR3wyqDUs5TL8cuk++odE4tSth1JHIyIDxsJCRDVOEASM6t4Qe6b6oGEdG9zJzMfQtTFYc/QG9HqOiIjoUSwsRCSZNm4q7J/hh0AvV5ToRfz3x8sY/81JZORyREREZbGwEJGk7KwsEDK8PRa/1hZKhQxHrjxAwHINTiRmSB2NiAwICwsRSU4QBIzoWh/h03zRuK4tUrQFGP51DEJ/u8YREREBYGEhIgPS0sUe+6f74bUObtCLwBeHrmLMphN4kK2TOhoRSYyFhYgMiq1SgWXD2uPzIe1gZSGD5loaAkI0iL6RJnU0IpIQCwsRGaQ3Ontg/3Q/NFPXwoNsHd5afxxf/XIVJRwREZklFhYiMlieajvsneaHoZ3doReBr365hrfWH0eqtkDqaERUw1hYiMigWVvKsXSIF/43zAs2lnLEJKQjIEQDzbUHUkcjohrEwkJERuHVDu7YP8MPLZztkJZTiNEbT+CLn6+guEQvdTQiqgEsLERkNJrUrYXwab4Y6V0fogiE/n4dI9cdx72sfKmjEVE1Y2EhIqNiZSHHolfbYsWIDqilVODEzQwELNfg98upUkcjomrEwkJERinQyxUHZvihjZs9HuYVYdzmk1gccQlFHBERmSQWFiIyWg2dbLF7ig/G+jQEAKw9loBha2NwJ5MjIiJTw8JCREZNqZBj4cDWWPNWR9hZKXA6KRMByzU4fPG+1NGIqAqxsBCRSXi5jQsigvzh5a5CVn4R3vk2Dp8cuIjCYo6IiEwBCwsRmQwPRxvsnOyDt/0aAQA2RCbijTXRSM7IkzgZET0vFhYiMimWChk+HNAK60Z3hsraAmdvZyEgRIOfzt+TOhoRPQcWFiIySX1aqRER7I+O9Wsju6AYk78/jQV7z0NXXCJ1NCKqBBYWIjJZbrWtsWNSd0zq0RgA8E3MLby+Oho303IlTkZEFcXCQkQmzUIuw7xXWmLT2C5wsLHA+TtaDFgRif1n70odjYgqgIWFiMxCrxb1EBHsj64NHZGjK8aMbWfw77BzKCjiiIjIGLCwEJHZcFFZY+s73pjxYlMIArD1eBIGr4zCjQc5UkcjomdgYSEis6KQy/Be3+b4dnxXONWyxOWUbASuiETYmdtSRyOip2BhISKz5O9ZFxFB/ujeuA7yCkvw7o6zmL3rLPILOSIiMkQsLERkturZW+H7Cd6Y2dsTggD8EHcbA0Mjce1+ttTRiOgfWFiIyKzJZQJm9m6GLRO8UddOiWupOQgMjcQPcckQRVHqeET0/7GwEBEB8GnihB+D/eHv6YSCIj1m7/oD7/1wFrm6YqmjERFYWIiISjnVUuKbcV3xr37NIROAPWfuYGBoJC7d00odjcjssbAQEf2NTCZgWq+m2D6xO5ztrXDjQS4Gr4zC1uNJHBERSYiFhYjoMbo2ckREsD96Nq8LXbEe/w47h6Dt8cguKJI6GpFZYmEhInoCR1tLbBzTBfNeaQG5TMD+s3cRuCIS5+9kSR2NyOywsBARPYVMJmBSjyb4YVJ3uNW2xs30PLy2KhrfxtzkiIioBrGwEBGVQ6cGDjgY5IfeLdUoLNHjo70XMG3raWTlc0REVBNYWIiIyqm2jSXWje6EDwe0goVcQMS5FAxYocHZ5EypoxGZPBYWIqIKEAQBb/s1wq7JPvBwtEZyRj6GrInGhshEjoiIqlGFC8uxY8cQGBgIV1dXCIKA8PDwMveLooiFCxfC1dUV1tbW6NmzJy5cuFDu82/fvh2CIGDw4MEVjUZEVGO8PGrjwAx/vNLGGUUlIj45cBETvzuFzLxCqaMRmaQKF5bc3Fx4eXkhNDT0sfcvXboUy5YtQ2hoKE6ePAlnZ2f06dMH2dnP/m2OW7du4f3334e/v39FYxER1TiVtQVWvdkR/xnUGpZyGQ5fvI/+IZE4nfRQ6mhEJkcQn+MapiAICAsLK70aIooiXF1dMXPmTMyZMwcAoNPpoFarsWTJEkyaNOmJ5yopKUGPHj0wbtw4aDQaZGZmPnL15mm0Wi1UKhWysrJgb29f2adERFQp5+9kYdrW07iVngeFTMC/+jXHO/6NIZMJUkcjMmjlff2u0vewJCYmIiUlBX379i3dp1Qq0aNHD0RHRz/12P/85z+oW7cu3n777XL9WTqdDlqttsxGRCSVNm4qHJjhhwHtXFCsF7H4x8uY8G0cMnI5IiKqClVaWFJSUgAAarW6zH61Wl163+NERUVhw4YNWLduXbn/rMWLF0OlUpVuHh4elQtNRFRF7KwssGJEByx6tS0sFTL8djkVAcs1OHkzQ+poREavWj4lJAhlL4GKovjIvr9kZ2fjrbfewrp16+Dk5FTuP2PevHnIysoq3ZKTk58rMxFRVRAEASO962PvNF80rmuLFG0Bhn8di5W/X4dez08REVWWoipP5uzsDODPKy0uLi6l+1NTUx+56vKXGzdu4ObNmwgMDCzdp9fr/wynUODKlSto0qTJI8cplUoolcqqjE9EVGVauthj/3Q/fBh+HnvO3MHnP19BbEI6/jesPZxq8d8uooqq0issjRo1grOzMw4fPly6r7CwEEePHoWPj89jj2nRogXOnTuH+Pj40m3gwIHo1asX4uPjOeohIqNlq1Tgy6FeWDqkHawsZNBcS0PAcg1ibqRLHY3I6FT4CktOTg6uX79eejsxMRHx8fFwdHRE/fr1MXPmTCxatAienp7w9PTEokWLYGNjg5EjR5YeM3r0aLi5uWHx4sWwsrJCmzZtyvwZtWvXBoBH9hMRGRtBEDC0swfae9TGtC2ncS01B2+uj0XQS56Y8aIn5PwUEVG5VLiwxMXFoVevXqW3Z82aBQAYM2YMNm/ejNmzZyM/Px9Tp07Fw4cP4e3tjUOHDsHOzq70mKSkJMhk/JJdIjIfzdR22DfdDwv2nccPcbfx1S/XcCIxA18Nb496dlZSxyMyeM/1PSyGhN/DQkTGYs/p25gffh55hSVwqmWJr4Z1gJ9n+T90QGRKJPkeFiIierbXOrpj33Q/tHC2Q1pOIUZtPI4vfr6C4hK91NGIDBYLCxGRBJrWq4Xwab4Y6V0fogiE/n4dI9cfR0pWgdTRiAwSCwsRkUSsLORY9GpbhIzogFpKBU4kZiAgRIMjV1KljkZkcFhYiIgkNtDLFftn+KG1qz0ycgsxdtNJ/PfHyyjiiIioFAsLEZEBaORki91TfDCmewMAwJqjNzD861jcycyXOBmRYWBhISIyEFYWcnw8qA1Wv9kRdlYKnLr1EP1DNPjl4n2poxFJjoWFiMjAvNLWBQdn+MPLXYXMvCJM+DYOnx64iMJijojIfLGwEBEZoPp1bLBzsg/G+zYCAKyPTMQba2OQnJEncTIiabCwEBEZKEuFDB8FtsK60Z2hsrbA2eRMBIRo8NP5e1JHI6pxLCxERAauTys1Dgb5oUP92sguKMbk709jwd7z0BWXSB2NqMawsBARGQF3Bxv8MKk7JvVoDAD4JuYWXl8djZtpuRInI6oZLCxEREbCQi7DvFdaYtPYLnCwscD5O1oMWBGJA3/clToaUbVjYSEiMjK9WtRDRLA/ujR0QI6uGNO3nsEHYedQUMQREZkuFhYiIiPkorLGtne6YXqvphAEYMvxJAxeGYUbD3KkjkZULVhYiIiMlEIuw/v9muPb8V1Rx9YSl1OyEbgiEuFn7kgdjajKsbAQERk5f8+6+DHYH90b10FeYQlm7ojHnF1/IL+QIyIyHSwsREQmoJ69Fb6f4I3glzwhCMCOuGQMWhmJa/ezpY5GVCVYWIiITIRcJuDdPs2w5W1v1LVT4ur9HAwMjcLOuGSpoxE9NxYWIiIT49PUCRFB/vD3dEJ+UQn+tesPzPohHrm6YqmjEVUaCwsRkQmqa6fEN+O64v2+zSATgD2n72BgaCQup2iljkZUKSwsREQmSiYTMP1FT2x7pxvU9krceJCLQaFR2HYiCaIoSh2PqEJYWIiITJx34zqICPJHz+Z1oSvWY96ecwjeHo8cjojIiLCwEBGZgTq1lNg4pgvmvdICcpmAfWfvYkCIBufvZEkdjahcWFiIiMyETCZgUo8m+GFSd7iqrHAzPQ+vrY7GdzE3OSIig8fCQkRkZjo1cEBEsD96t1SjsFiPD/dewLStp6EtKJI6GtETsbAQEZmh2jaWWDe6E+b3bwkLuYCIcykYEBKJP25nSh2N6LFYWIiIzJQgCJjg3xg7J/vA3cEaSRl5eH11NDZGJnJERAaHhYWIyMy196iNg0H+eLm1M4pKRPznwEVM+u4UsvI4IiLDwcJCRERQWVtg9Vsd8fHA1rCUy3Do4n0EhGhwOumh1NGIALCwEBHR/ycIAsb4NMSeqT5oUMcGdzLzMXRNDL4+dgN6PUdEJC0WFiIiKqONmwoHZvhhQDsXFOtFLIq4jAnfxuFhbqHU0ciMsbAQEdEj7KwssGJEByx6tS0sFTL8djkVASEanLyZIXU0MlMsLERE9FiCIGCkd33sneaLxk62uJdVgOFfx2Ll79c5IqIax8JCRERP1dLFHvtn+OHVDm4o0Yv4/OcrGLv5JNJydFJHIzPCwkJERM9kq1Rg2VAvLH29HawsZDh29QEClmsQm5AudTQyEywsRERULoIgYGgXD+yb7gfPerWQmq3DyHWxWP7LNZRwRETVjIWFiIgqpJnaDnun++KNTu7Qi8D/frmK0RuPIzW7QOpoZMJYWIiIqMJsLBX4/A0vLBvqBRtLOaKupyNgeSQir6VJHY1MFAsLERFV2msd3bFvuh9aONshLUeHURuP48tDV1Bcopc6GpkYFhYiInouTevVQvg0X4zoWh+iCKz47TpGrj+OlCyOiKjqsLAQEdFzs7KQY/FrbREyogNsLeU4kZiBgBANjlxJlToamQgWFiIiqjIDvVxxIMgfrV3tkZFbiLGbTmLJT5dRxBERPScWFiIiqlKNnGyxe4oPRndvAABYfeQGhn8di7uZ+RInI2PGwkJERFXOykKO/wxqg1VvdoSdUoFTtx4iIESDXy/dlzoaGSkWFiIiqjYBbV1wMMgf7dxVyMwrwtvfxOHTAxdRWMwREVUMCwsREVWr+nVssGuyD8b7NgIArI9MxNC1MUjOyJM4GRkTFhYiIqp2lgoZPgpsha9HdYK9lQLxyZnoH6LBzxdSpI5GRoKFhYiIakzf1s6ICPZHh/q1oS0oxqTvTmHhvgvQFZdIHY0MHAsLERHVKHcHG/wwqTsmvdAYALA5+iaGrI7BrfRciZORIWNhISKiGmchl2FeQEtsHNsZDjYWOHcnCwNCInHwj3tSRyMDxcJCRESSebGFGhHB/ujS0AHZumJM23oa88PPoaCIIyIqi4WFiIgk5aKyxrZ3umFaryYQBOD72CS8uioaCQ9ypI5GBoSFhYiIJKeQy/Cvfi3wzbiuqGNriUv3tAhcEYm98XekjkYGgoWFiIgMxgvN6iIi2B/dGjsit7AEwdvjMXf3H8gv5IjI3LGwEBGRQVHbW2HLhG4IfskTggBsP5mMwSujcD01W+poJCEWFiIiMjhymYB3+zTDlre9UddOiSv3sxG4Igq7Tt2WOhpJpMKF5dixYwgMDISrqysEQUB4eHiZ+0VRxMKFC+Hq6gpra2v07NkTFy5ceOo5161bB39/fzg4OMDBwQG9e/fGiRMnKhqNiIhMjE9TJ0QE+cOvqRPyi0rw/s6zeO+Hs8grLJY6GtWwCheW3NxceHl5ITQ09LH3L126FMuWLUNoaChOnjwJZ2dn9OnTB9nZT76Ud+TIEYwYMQK///47YmJiUL9+ffTt2xd37vDNVkRE5q6unRLfjO+K9/s2g0wAdp++jcAVkbiSwhGRORFEURQrfbAgICwsDIMHDwbw59UVV1dXzJw5E3PmzAEA6HQ6qNVqLFmyBJMmTSrXeUtKSuDg4IDQ0FCMHj26XMdotVqoVCpkZWXB3t6+Us+HiIgM2/GEdARtP4P7Wh2UChk+Htgaw7p4QBAEqaNRJZX39btK38OSmJiIlJQU9O3bt3SfUqlEjx49EB0dXe7z5OXloaioCI6Ojk98jE6ng1arLbMREZFp825cBxFB/ujRrC50xXrM3XMOM3fEI0fHEZGpq9LCkpLy569uqtXqMvvVanXpfeUxd+5cuLm5oXfv3k98zOLFi6FSqUo3Dw+PyoUmIiKjUqeWEpvGdsHcV1pALhOwN/4uAldE4sLdLKmjUTWqlk8J/fPSnCiK5b5ct3TpUmzbtg179uyBlZXVEx83b948ZGVllW7JycnPlZmIiIyHTCZgco8m+GFSN7iqrJCYlotXV0Xju9hbeI53OpABq9LC4uzsDACPXE1JTU195KrL43zxxRdYtGgRDh06hHbt2j31sUqlEvb29mU2IiIyL50aOOJgkD96t6yHwmI9Pgw/j+lbz0BbUCR1NKpiVVpYGjVqBGdnZxw+fLh0X2FhIY4ePQofH5+nHvv555/jk08+wU8//YTOnTtXZSwiIjJhDraWWDe6M+b3bwmFTMDBc/cwICQSf9zOlDoaVaEKF5acnBzEx8cjPj4ewJ9vtI2Pj0dSUhIEQcDMmTOxaNEihIWF4fz58xg7dixsbGwwcuTI0nOMHj0a8+bNK729dOlSzJ8/Hxs3bkTDhg2RkpKClJQU5OTwh6+IiOjZBEHABP/G2DXFB+4O1kjKyMPrq6OxKSqRIyITUeGPNR85cgS9evV6ZP+YMWOwefNmiKKIjz/+GGvXrsXDhw/h7e2NlStXok2bNqWP7dmzJxo2bIjNmzcDABo2bIhbt249cs4FCxZg4cKF5crFjzUTEREAZOUXYfaus/j5wn0AQL/Waix93QsqGwuJk9HjlPf1+7m+h8WQsLAQEdFfRFHEtzG38NnBSygs0cOttjVCR3ZAh/oOUkejf5Dke1iIiIgMgSAIGOPTELun+KBBHRvcyczHG2tisO5YAkdERoqFhYiITFZbdxUOzPBD/3YuKNaL+CziEiZ8E4eHuYVSR6MKYmEhIiKTZmdlgdARHfDZq21gqZDh18up6B+iQdzNDKmjUQWwsBARkckTBAFvejdA+FRfNHayxd2sAgz7OharjlyHXs8RkTFgYSEiIrPRytUe+2b4YXB7V5ToRSz96QrGbT6J9Byd1NHoGVhYiIjIrNRSKvC/Ye2x9PV2sLKQ4ejVBwgI0SA2IV3qaPQULCxERGR2BEHA0C4e2DvND03r1cJ9rQ4j18Ui5NdrKOGIyCCxsBARkdlq7myHfdN9MaSTO/QisOzwVYzeeByp2QVSR6N/YGEhIiKzZmOpwBdveOHLN7xgbSFH1PV0BCyPRNT1NKmj0d+wsBAREQF4vZM79s/wQ3O1HdJydHhrw3EsO3yVIyIDwcJCRET0/zWtVwt7p/tiRFcPiCIQ8us1jFwXi/tajoikxsJCRET0N1YWcix+rR2WD28PW0s5jidmIGC5BkevPpA6mlljYSEiInqMQe3dcCDIH61c7JGeW4gxG09gyU+XUVyilzqaWWJhISIieoJGTrbYM9UHo7o1AACsPnIDw7+Oxd3MfImTmR8WFiIioqewspDjk8FtsOrNjrBTKhB36yECQjT47fJ9qaOZFRYWIiKicgho64KDQf5o565CZl4Rxm+Ow2cHL6KII6IawcJCRERUTvXr2GDn5O4Y59sQALBOk4g31sQgOSNP2mBmgIWFiIioApQKORYEtsbaUZ1gb6VAfHIm+odo8POFFKmjmTQWFiIiokro19oZEcH+aO9RG9qCYkz67hQ+3n8BuuISqaOZJBYWIiKiSnJ3+HNENPGFxgCATVE3MWR1DJLSOSKqaiwsREREz8FCLsO/A1pi49jOcLCxwLk7WegfokHEuXtSRzMpLCxERERV4MUWakQE+6NzAwdk64oxdctpfBh+HgVFHBFVBRYWIiKiKuKissb2id0wtWcTAMB3sbfw2qpoJKblSpzM+LGwEBERVSGFXIbZL7fAN+O7oo6tJS7e02JAiAZ74+9IHc2osbAQERFVgx7N6iIi2B/dGjsit7AEwdvjMXf3HxwRVRILCxERUTVR21thy4RuCHrJE4IAbD+ZjEGhUbiemi11NKPDwkJERFSN5DIBs/o0w/dve8OplhJX7mcjcEUUdp+6LXU0o8LCQkREVAN8mzohItgPvk3rIL+oBO/tPIv3d55FXmGx1NGMAgsLERFRDalnZ4Vvx3vjvT7NIBOAXaduY2BoFK6kcET0LCwsRERENUguEzDjJU9sfacb1PZKXE/NwaCVkdhxMgmiKEodz2CxsBAREUmgW+M6iAjyR49mdVFQpMec3efw7o545Og4InocFhYiIiKJ1KmlxKaxXTDn5RaQywSEx9/FwBWRuHhXK3U0g8PCQkREJCGZTMCUnk2wY2I3uKiskJCWi8GrovB97C2OiP6GhYWIiMgAdG7oiIggf7zUoh4Ki/WYH34e07edQXZBkdTRDAILCxERkYFwsLXE+jGdMb9/SyhkAg7+cQ8DVkTi3O0sqaNJjoWFiIjIgAiCgAn+jbFzcne41bbGrfQ8vL46GpujEs16RMTCQkREZIA61HdARJA/+rZSo7BEj4X7L2Ly96eQlWeeIyIWFiIiIgOlsrHA2lGdsDCwFSzlMvx84T76r9AgPjlT6mg1joWFiIjIgAmCgLG+jbB7ig/qO9rg9sN8DFkdjfWaBLMaEbGwEBERGYG27iocCPJD/3YuKNaL+PTgJbzzbRwy8wqljlYjWFiIiIiMhL2VBUJHdMCng9vAUiHDL5dSEbBcg1O3MqSOVu1YWIiIiIyIIAh4q1sDhE31QSMnW9zNKsDQtbFYfeQG9HrTHRGxsBARERmh1q4q7J/hh0HtXVGiF7Hkp8sY/81JpOfopI5WLVhYiIiIjFQtpQJfDWuPJa+3hVIhw5ErDxAQosHxhHSpo1U5FhYiIiIjJggChnWpj33T/dC0Xi3c1+owYl0sVvx6DSUmNCJiYSEiIjIBzZ3tsG+6L17v6A69CHx5+CrGbDyBB9mmMSJiYSEiIjIRNpYKfDnUC1+84QVrCzkir6fhleUaRF9Pkzrac2NhISIiMjFDOrlj/wxfNFfbIS1Hhzc3HMeyw1eNekTEwkJERGSCmtazw97pvhjR1QOiCIT8eg1vro/FfW2B1NEqhYWFiIjIRFlZyLH4tXZYPrw9bC3liE3IQMByDY5dfSB1tApjYSEiIjJxg9q7Yf8MP7R0sUd6biFGbzyBpT9dRnGJXupo5cbCQkREZAYa162FsKk+GNWtAQBg1ZEbGLEuFvey8iVOVj4sLERERGbCykKOTwa3wcqRHWGnVODkzYcIWK7B75dTpY72TCwsREREZqZ/OxccCPJDWzcVHuYVYdzmk1gccQlFBjwiYmEhIiIyQw3q2GLXlO4Y69MQALD2WAKGro3B7Yd50gZ7AhYWIiIiM6VUyLFwYGusHdUJ9lYKnEnKRMByDQ5dSJE62iMqXFiOHTuGwMBAuLq6QhAEhIeHl7lfFEUsXLgQrq6usLa2Rs+ePXHhwoVnnnf37t1o1aoVlEolWrVqhbCwsIpGIyIiokro19oZB4P80d6jNrQFxZj43Sl8vP8CCosNZ0RU4cKSm5sLLy8vhIaGPvb+pUuXYtmyZQgNDcXJkyfh7OyMPn36IDs7+4nnjImJwbBhwzBq1CicPXsWo0aNwtChQ3H8+PGKxiMiIqJK8HC0wQ+TuuMd/0YAgE1RNzFkTTSS0g1jRCSIoljp7+kVBAFhYWEYPHgwgD+vrri6umLmzJmYM2cOAECn00GtVmPJkiWYNGnSY88zbNgwaLVa/Pjjj6X7Xn75ZTg4OGDbtm3lyqLVaqFSqZCVlQV7e/vKPiUiIiKz9+ul+3hv51lk5hXBTqnAkiHtENDWpVr+rPK+flfpe1gSExORkpKCvn37lu5TKpXo0aMHoqOjn3hcTExMmWMAoF+/fk89RqfTQavVltmIiIjo+b3UUo2IIH90buCAbF0xpm45jQ/Dz6OgqESyTFVaWFJS/nyTjlqtLrNfrVaX3vek4yp6zOLFi6FSqUo3Dw+P50hOREREf+da2xrbJnbD1J5NAADfxd7C3vg7kuWplk8JCYJQ5rYoio/se95j5s2bh6ysrNItOTm58oGJiIjoERZyGWa/3ALfjO+K1zq64Y1O0l0cUFTlyZydnQH8ecXExeX/Zl2pqamPXEH553H/vJryrGOUSiWUSuVzJiYiIqJn6dGsLno0qytphiq9wtKoUSM4Ozvj8OHDpfsKCwtx9OhR+Pj4PPG47t27lzkGAA4dOvTUY4iIiMh8VPgKS05ODq5fv156OzExEfHx8XB0dET9+vUxc+ZMLFq0CJ6envD09MSiRYtgY2ODkSNHlh4zevRouLm5YfHixQCA4OBgvPDCC1iyZAkGDRqEvXv34pdffkFkZGQVPEUiIiIydhUuLHFxcejVq1fp7VmzZgEAxowZg82bN2P27NnIz8/H1KlT8fDhQ3h7e+PQoUOws7MrPSYpKQky2f9d3PHx8cH27dsxf/58fPjhh2jSpAl27NgBb2/v53luREREZCKe63tYDAm/h4WIiMj4SPI9LERERETVgYWFiIiIDB4LCxERERk8FhYiIiIyeCwsREREZPBYWIiIiMjgsbAQERGRwWNhISIiIoPHwkJEREQGr0p/rVlKf31hr1arlTgJERERlddfr9vP+uJ9kyks2dnZAAAPDw+JkxAREVFFZWdnQ6VSPfF+k/ktIb1ej7t378LOzg6CIFTZebVaLTw8PJCcnMzfKKpGXOeaw7WuGVznmsF1rhnVuc6iKCI7Oxuurq5lfhj5n0zmCotMJoO7u3u1nd/e3p7/MdQArnPN4VrXDK5zzeA614zqWuenXVn5C990S0RERAaPhYWIiIgMHgvLMyiVSixYsABKpVLqKCaN61xzuNY1g+tcM7jONcMQ1tlk3nRLREREpotXWIiIiMjgsbAQERGRwWNhISIiIoPHwkJEREQGj4UFwKpVq9CoUSNYWVmhU6dO0Gg0T3380aNH0alTJ1hZWaFx48ZYs2ZNDSU1bhVZ5z179qBPnz6oW7cu7O3t0b17d/z88881mNZ4VfTv81+ioqKgUCjQvn376g1oQiq61jqdDh988AEaNGgApVKJJk2aYOPGjTWU1nhVdJ23bNkCLy8v2NjYwMXFBePGjUN6enoNpTVOx44dQ2BgIFxdXSEIAsLDw595TI2/Fopmbvv27aKFhYW4bt068eLFi2JwcLBoa2sr3rp167GPT0hIEG1sbMTg4GDx4sWL4rp160QLCwtx165dNZzcuFR0nYODg8UlS5aIJ06cEK9evSrOmzdPtLCwEE+fPl3DyY1LRdf5L5mZmWLjxo3Fvn37il5eXjUT1shVZq0HDhwoent7i4cPHxYTExPF48ePi1FRUTWY2vhUdJ01Go0ok8nE5cuXiwkJCaJGoxFbt24tDh48uIaTG5eIiAjxgw8+EHfv3i0CEMPCwp76eCleC82+sHTt2lWcPHlymX0tWrQQ586d+9jHz549W2zRokWZfZMmTRK7detWbRlNQUXX+XFatWolfvzxx1UdzaRUdp2HDRsmzp8/X1ywYAELSzlVdK1//PFHUaVSienp6TURz2RUdJ0///xzsXHjxmX2hYSEiO7u7tWW0dSUp7BI8Vpo1iOhwsJCnDp1Cn379i2zv2/fvoiOjn7sMTExMY88vl+/foiLi0NRUVG1ZTVmlVnnf9Lr9cjOzoajo2N1RDQJlV3nTZs24caNG1iwYEF1RzQZlVnrffv2oXPnzli6dCnc3NzQrFkzvP/++8jPz6+JyEapMuvs4+OD27dvIyIiAqIo4v79+9i1axf69+9fE5HNhhSvhSbz44eVkZaWhpKSEqjV6jL71Wo1UlJSHntMSkrKYx9fXFyMtLQ0uLi4VFteY1WZdf6nL7/8Erm5uRg6dGh1RDQJlVnna9euYe7cudBoNFAozPqfgwqpzFonJCQgMjISVlZWCAsLQ1paGqZOnYqMjAy+j+UJKrPOPj4+2LJlC4YNG4aCggIUFxdj4MCBWLFiRU1ENhtSvBaa9RWWvwiCUOa2KIqP7HvW4x+3n8qq6Dr/Zdu2bVi4cCF27NiBevXqVVc8k1HedS4pKcHIkSPx8ccfo1mzZjUVz6RU5O+0Xq+HIAjYsmULunbtioCAACxbtgybN2/mVZZnqMg6X7x4EUFBQfjoo49w6tQp/PTTT0hMTMTkyZNrIqpZqenXQrP+XyonJyfI5fJHmnpqauojzfEvzs7Oj328QqFAnTp1qi2rMavMOv9lx44dePvtt7Fz50707t27OmMavYquc3Z2NuLi4nDmzBlMnz4dwJ8vqqIoQqFQ4NChQ3jxxRdrJLuxqczfaRcXF7i5uUGlUpXua9myJURRxO3bt+Hp6VmtmY1RZdZ58eLF8PX1xb/+9S8AQLt27WBrawt/f398+umnvApeRaR4LTTrKyyWlpbo1KkTDh8+XGb/4cOH4ePj89hjunfv/sjjDx06hM6dO8PCwqLashqzyqwz8OeVlbFjx2Lr1q2cP5dDRdfZ3t4e586dQ3x8fOk2efJkNG/eHPHx8fD29q6p6EanMn+nfX19cffuXeTk5JTuu3r1KmQyGdzd3as1r7GqzDrn5eVBJiv70iaXywH83xUAen6SvBZW29t5jcRfH5nbsGGDePHiRXHmzJmira2tePPmTVEURXHu3LniqFGjSh//10e53n33XfHixYvihg0b+LHmcqjoOm/dulVUKBTiypUrxXv37pVumZmZUj0Fo1DRdf4nfkqo/Cq61tnZ2aK7u7s4ZMgQ8cKFC+LRo0dFT09PccKECVI9BaNQ0XXetGmTqFAoxFWrVok3btwQIyMjxc6dO4tdu3aV6ikYhezsbPHMmTPimTNnRADismXLxDNnzpR+fNwQXgvNvrCIoiiuXLlSbNCggWhpaSl27NhRPHr0aOl9Y8aMEXv06FHm8UeOHBE7dOggWlpaig0bNhRXr15dw4mNU0XWuUePHiKAR7YxY8bUfHAjU9G/z3/HwlIxFV3rS5cuib179xatra1Fd3d3cdasWWJeXl4NpzY+FV3nkJAQsVWrVqK1tbXo4uIivvnmm+Lt27drOLVx+f3335/6b64hvBYKoshrZERERGTYzPo9LERERGQcWFiIiIjI4LGwEBERkcFjYSEiIiKDx8JCREREBo+FhYiIiAweCwsREREZPBYWIiIiMngsLERERGTwWFiIiIjI4LGwEBERkcFjYSEiIiKD9/8AeRkQ5sfBYqUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(eps_rewards)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
